{
 "cells": [
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:45:59.270516Z",
     "iopub.status.busy": "2025-01-21T06:45:59.270253Z",
     "iopub.status.idle": "2025-01-21T06:46:01.389869Z",
     "shell.execute_reply": "2025-01-21T06:46:01.389332Z",
     "shell.execute_reply.started": "2025-01-21T06:45:59.270496Z"
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:54.203660Z",
     "start_time": "2025-03-07T09:44:54.104378Z"
    }
   },
   "source": [
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import pandas as pd\n",
    "import os\n",
    "import sys\n",
    "import time\n",
    "from tqdm.auto import tqdm\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "print(sys.version_info)\n",
    "for module in mpl, np, pd, sklearn, torch:\n",
    "    print(module.__name__, module.__version__)\n",
    "    \n",
    "device = torch.device(\"cuda:0\") if torch.cuda.is_available() else torch.device(\"cpu\")\n",
    "print(device)\n",
    "\n",
    "seed = 42\n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sys.version_info(major=3, minor=12, micro=3, releaselevel='final', serial=0)\n",
      "matplotlib 3.10.0\n",
      "numpy 1.26.4\n",
      "pandas 2.2.3\n",
      "sklearn 1.6.1\n",
      "torch 2.6.0+cu118\n",
      "cuda:0\n"
     ]
    }
   ],
   "execution_count": 22
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 数据准备\n",
    "\n",
    "https://www.kaggle.com/competitions/cifar-10/data\n",
    "\n",
    "```shell\n",
    "$ tree -L 1 cifar-10                                    \n",
    "cifar-10\n",
    "├── sampleSubmission.csv\n",
    "├── test\n",
    "├── train\n",
    "└── trainLabels.csv\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:01.391389Z",
     "iopub.status.busy": "2025-01-21T06:46:01.390831Z",
     "iopub.status.idle": "2025-01-21T06:46:01.393562Z",
     "shell.execute_reply": "2025-01-21T06:46:01.393101Z",
     "shell.execute_reply.started": "2025-01-21T06:46:01.391365Z"
    },
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:54.209259Z",
     "start_time": "2025-03-07T09:44:54.204669Z"
    }
   },
   "source": [
    "# %pip install py7zr\n",
    "# import py7zr\n",
    "# a = py7zr.SevenZipFile(r'../input/cifar-10/train.7z','r')\n",
    "# a.extractall(path=r'./')\n",
    "# a.close()"
   ],
   "outputs": [],
   "execution_count": 23
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:01.394307Z",
     "iopub.status.busy": "2025-01-21T06:46:01.394134Z",
     "iopub.status.idle": "2025-01-21T06:46:01.511635Z",
     "shell.execute_reply": "2025-01-21T06:46:01.511010Z",
     "shell.execute_reply.started": "2025-01-21T06:46:01.394289Z"
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:54.453152Z",
     "start_time": "2025-03-07T09:44:54.210214Z"
    }
   },
   "source": [
    "#我们不运行test数据集，太多图片\n",
    "!ls ."
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cifar-10\n",
      "new_inception_net.ipynb\n",
      "new_resnet.ipynb\n",
      "new-vgg-aliyun.ipynb\n"
     ]
    }
   ],
   "execution_count": 24
  },
  {
   "cell_type": "code",
   "metadata": {
    "ExecutionIndicator": {
     "show": true
    },
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:01.630423Z",
     "iopub.status.busy": "2025-01-21T06:46:01.629953Z",
     "iopub.status.idle": "2025-01-21T06:46:03.912703Z",
     "shell.execute_reply": "2025-01-21T06:46:03.912046Z",
     "shell.execute_reply.started": "2025-01-21T06:46:01.630390Z"
    },
    "tags": [],
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:56.510410Z",
     "start_time": "2025-03-07T09:44:54.454550Z"
    }
   },
   "source": [
    "from pathlib import Path\n",
    "\n",
    "DATA_DIR = Path(\".\")\n",
    "DATA_DIR1 =Path(\"cifar-10\")\n",
    "train_lables_file = DATA_DIR1 / \"trainLabels.csv\"\n",
    "test_csv_file = DATA_DIR1 / \"sampleSubmission.csv\" #测试集模板csv文件\n",
    "train_folder = DATA_DIR1 / \"train\"\n",
    "test_folder = DATA_DIR1 / \"test\"\n",
    "\n",
    "#所有的类别\n",
    "class_names = [\n",
    "    'airplane',\n",
    "    'automobile',\n",
    "    'bird',\n",
    "    'cat',\n",
    "    'deer',\n",
    "    'dog',\n",
    "    'frog',\n",
    "    'horse',\n",
    "    'ship',\n",
    "    'truck',\n",
    "]\n",
    "\n",
    "def parse_csv_file(filepath, folder):\n",
    "    \"\"\"Parses csv files into (filename(path), label) format\"\"\"\n",
    "    results = []\n",
    "    #读取所有行\n",
    "    with open(filepath, 'r') as f:\n",
    "#         lines = f.readlines()  为什么加[1:]，可以试这个\n",
    "        #第一行不需要，因为第一行是标签\n",
    "        lines = f.readlines()[1:] \n",
    "    for line in lines:#依次去取每一行\n",
    "        image_id, label_str = line.strip('\\n').split(',')\n",
    "        image_full_path = folder / f\"{image_id}.png\"\n",
    "        results.append((image_full_path, label_str)) #得到对应图片的路径和分类\n",
    "    return results\n",
    "\n",
    "#解析对应的文件夹\n",
    "train_labels_info = parse_csv_file(train_lables_file, train_folder)\n",
    "test_csv_info = parse_csv_file(test_csv_file, test_folder)\n",
    "#打印\n",
    "import pprint\n",
    "pprint.pprint(train_labels_info[0:5])\n",
    "pprint.pprint(test_csv_info[0:5])\n",
    "print(len(train_labels_info), len(test_csv_info))"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(WindowsPath('cifar-10/train/1.png'), 'frog'),\n",
      " (WindowsPath('cifar-10/train/2.png'), 'truck'),\n",
      " (WindowsPath('cifar-10/train/3.png'), 'truck'),\n",
      " (WindowsPath('cifar-10/train/4.png'), 'deer'),\n",
      " (WindowsPath('cifar-10/train/5.png'), 'automobile')]\n",
      "[(WindowsPath('cifar-10/test/1.png'), 'cat'),\n",
      " (WindowsPath('cifar-10/test/2.png'), 'cat'),\n",
      " (WindowsPath('cifar-10/test/3.png'), 'cat'),\n",
      " (WindowsPath('cifar-10/test/4.png'), 'cat'),\n",
      " (WindowsPath('cifar-10/test/5.png'), 'cat')]\n",
      "50000 300000\n"
     ]
    }
   ],
   "execution_count": 25
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:03.914011Z",
     "iopub.status.busy": "2025-01-21T06:46:03.913478Z",
     "iopub.status.idle": "2025-01-21T06:46:03.971502Z",
     "shell.execute_reply": "2025-01-21T06:46:03.970860Z",
     "shell.execute_reply.started": "2025-01-21T06:46:03.913976Z"
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:56.612519Z",
     "start_time": "2025-03-07T09:44:56.511857Z"
    }
   },
   "source": [
    "# train_df = pd.DataFrame(train_labels_info)\n",
    "train_df = pd.DataFrame(train_labels_info[0:45000])\n",
    "valid_df = pd.DataFrame(train_labels_info[45000:])\n",
    "test_df = pd.DataFrame(test_csv_info)\n",
    "\n",
    "train_df.columns = ['filepath', 'class']\n",
    "valid_df.columns = ['filepath', 'class']\n",
    "test_df.columns = ['filepath', 'class']\n",
    "\n",
    "print(train_df.head())\n",
    "print(valid_df.head())\n",
    "print(test_df.head())"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "               filepath       class\n",
      "0  cifar-10\\train\\1.png        frog\n",
      "1  cifar-10\\train\\2.png       truck\n",
      "2  cifar-10\\train\\3.png       truck\n",
      "3  cifar-10\\train\\4.png        deer\n",
      "4  cifar-10\\train\\5.png  automobile\n",
      "                   filepath       class\n",
      "0  cifar-10\\train\\45001.png       horse\n",
      "1  cifar-10\\train\\45002.png  automobile\n",
      "2  cifar-10\\train\\45003.png        deer\n",
      "3  cifar-10\\train\\45004.png  automobile\n",
      "4  cifar-10\\train\\45005.png    airplane\n",
      "              filepath class\n",
      "0  cifar-10\\test\\1.png   cat\n",
      "1  cifar-10\\test\\2.png   cat\n",
      "2  cifar-10\\test\\3.png   cat\n",
      "3  cifar-10\\test\\4.png   cat\n",
      "4  cifar-10\\test\\5.png   cat\n"
     ]
    }
   ],
   "execution_count": 26
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:03.972824Z",
     "iopub.status.busy": "2025-01-21T06:46:03.972308Z",
     "iopub.status.idle": "2025-01-21T06:46:04.781633Z",
     "shell.execute_reply": "2025-01-21T06:46:04.781099Z",
     "shell.execute_reply.started": "2025-01-21T06:46:03.972790Z"
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:59.013220Z",
     "start_time": "2025-03-07T09:44:56.613528Z"
    }
   },
   "source": [
    "from PIL import Image\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "from torchvision import transforms\n",
    "\n",
    "class Cifar10Dataset(Dataset):\n",
    "    df_map = {\n",
    "        \"train\": train_df,\n",
    "        \"eval\": valid_df,\n",
    "        \"test\": test_df\n",
    "    }\n",
    "    label_to_idx = {label: idx for idx, label in enumerate(class_names)}\n",
    "    idx_to_label = {idx: label for idx, label in enumerate(class_names)}\n",
    "    def __init__(self, mode, transform=None):\n",
    "        self.df = self.df_map.get(mode, None)\n",
    "        if self.df is None:\n",
    "            raise ValueError(\"mode should be one of train, val, test, but got {}\".format(mode))\n",
    "\n",
    "        self.transform = transform\n",
    "        \n",
    "    def __getitem__(self, index):\n",
    "        img_path, label = self.df.iloc[index]\n",
    "        img = Image.open(img_path).convert('RGB')\n",
    "        # # img 转换为 channel first\n",
    "        # img = img.transpose((2, 0, 1))\n",
    "        # transform\n",
    "        img = self.transform(img)\n",
    "        # label 转换为 idx\n",
    "        label = self.label_to_idx[label]\n",
    "        return img, label\n",
    "    \n",
    "    def __len__(self):\n",
    "        return self.df.shape[0]\n",
    "    \n",
    "IMAGE_SIZE = 32\n",
    "mean, std = [0.4914, 0.4822, 0.4465], [0.247, 0.243, 0.261]\n",
    "\n",
    "transforms_train = transforms.Compose([\n",
    "        # resize\n",
    "        transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n",
    "        # random rotation 40\n",
    "        transforms.RandomRotation(40),\n",
    "        # horizaontal flip\n",
    "        transforms.RandomHorizontalFlip(),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean, std)\n",
    "    ])\n",
    "\n",
    "transforms_eval = transforms.Compose([\n",
    "        # resize\n",
    "        transforms.Resize((IMAGE_SIZE, IMAGE_SIZE)),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize(mean, std)\n",
    "    ])\n",
    "\n",
    "train_ds = Cifar10Dataset(\"train\", transforms_train)\n",
    "eval_ds = Cifar10Dataset(\"eval\", transforms_eval) "
   ],
   "outputs": [],
   "execution_count": 27
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.782891Z",
     "iopub.status.busy": "2025-01-21T06:46:04.782344Z",
     "iopub.status.idle": "2025-01-21T06:46:04.785961Z",
     "shell.execute_reply": "2025-01-21T06:46:04.785438Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.782864Z"
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:59.017733Z",
     "start_time": "2025-03-07T09:44:59.014228Z"
    }
   },
   "source": [
    "batch_size = 64\n",
    "train_dl = DataLoader(train_ds, batch_size=batch_size, shuffle=True, num_workers=4)   \n",
    "eval_dl = DataLoader(eval_ds, batch_size=batch_size, shuffle=False, num_workers=4)"
   ],
   "outputs": [],
   "execution_count": 28
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.786836Z",
     "iopub.status.busy": "2025-01-21T06:46:04.786615Z",
     "iopub.status.idle": "2025-01-21T06:46:04.789337Z",
     "shell.execute_reply": "2025-01-21T06:46:04.788884Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.786817Z"
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:59.021524Z",
     "start_time": "2025-03-07T09:44:59.018244Z"
    }
   },
   "source": [
    "# 遍历train_ds得到每张图片，计算每个通道的均值和方差\n",
    "#这里不需要每次都执行，只需要执行一次就可以了，所以注释掉\n",
    "# def cal_mean_std(ds):\n",
    "#     mean = 0.\n",
    "#     std = 0.\n",
    "#     for img, _ in ds:\n",
    "#         mean += img.mean(dim=(1, 2))\n",
    "#         std += img.std(dim=(1, 2))\n",
    "#     mean /= len(ds)\n",
    "#     std /= len(ds)\n",
    "#     return mean, std\n",
    "#\n",
    "# # 经过 normalize 后 均值为0，方差为1\n",
    "# print(cal_mean_std(train_ds))"
   ],
   "outputs": [],
   "execution_count": 29
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 定义模型"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.790020Z",
     "iopub.status.busy": "2025-01-21T06:46:04.789854Z",
     "iopub.status.idle": "2025-01-21T06:46:04.800044Z",
     "shell.execute_reply": "2025-01-21T06:46:04.799590Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.790003Z"
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:59.038597Z",
     "start_time": "2025-03-07T09:44:59.022535Z"
    }
   },
   "source": [
    "class VGG(nn.Module):\n",
    "    def __init__(self, num_classes):\n",
    "        super().__init__()\n",
    "        self.model = nn.Sequential(\n",
    "            nn.Conv2d(in_channels=3, out_channels=32, kernel_size=3, padding=\"same\"),# 32*32*32\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, padding=\"same\"),# 32*32*32\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2),\n",
    "            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, padding=\"same\"),# 32*16*16\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, padding=\"same\"),# 32*16*16\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2),\n",
    "            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, padding=\"same\"),# 32*8*8\n",
    "            nn.ReLU(),\n",
    "            nn.Conv2d(in_channels=32, out_channels=32, kernel_size=3, padding=\"same\"),# 32*8*8\n",
    "            nn.ReLU(),\n",
    "            nn.MaxPool2d(kernel_size=2), # 32*4*4\n",
    "            nn.Flatten(),\n",
    "            nn.Linear(512, num_classes),\n",
    "        )\n",
    "        self.init_weights()\n",
    "        \n",
    "    def init_weights(self):\n",
    "        \"\"\"使用 xavier 均匀分布来初始化全连接层、卷积层的权重 W\"\"\"\n",
    "        for m in self.modules():\n",
    "            if isinstance(m, (nn.Linear, nn.Conv2d)):\n",
    "                nn.init.xavier_uniform_(m.weight)\n",
    "                nn.init.zeros_(m.bias)\n",
    "        \n",
    "    def forward(self, x):\n",
    "        return self.model(x)\n",
    "        \n",
    "for key, value in VGG(len(class_names)).named_parameters():\n",
    "    print(f\"{key:^40}paramerters num: {np.prod(value.shape)}\")\n",
    "    \n"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             model.0.weight             paramerters num: 864\n",
      "              model.0.bias              paramerters num: 32\n",
      "             model.2.weight             paramerters num: 9216\n",
      "              model.2.bias              paramerters num: 32\n",
      "             model.5.weight             paramerters num: 9216\n",
      "              model.5.bias              paramerters num: 32\n",
      "             model.7.weight             paramerters num: 9216\n",
      "              model.7.bias              paramerters num: 32\n",
      "            model.10.weight             paramerters num: 9216\n",
      "             model.10.bias              paramerters num: 32\n",
      "            model.12.weight             paramerters num: 9216\n",
      "             model.12.bias              paramerters num: 32\n",
      "            model.16.weight             paramerters num: 5120\n",
      "             model.16.bias              paramerters num: 10\n"
     ]
    }
   ],
   "execution_count": 30
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.800691Z",
     "iopub.status.busy": "2025-01-21T06:46:04.800510Z",
     "iopub.status.idle": "2025-01-21T06:46:04.804608Z",
     "shell.execute_reply": "2025-01-21T06:46:04.804173Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.800673Z"
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:59.044440Z",
     "start_time": "2025-03-07T09:44:59.039222Z"
    }
   },
   "source": [
    "32*4*4"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "512"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 31
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 训练\n",
    "\n",
    "pytorch的训练需要自行实现，包括\n",
    "1. 定义损失函数\n",
    "2. 定义优化器\n",
    "3. 定义训练步\n",
    "4. 训练"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.805417Z",
     "iopub.status.busy": "2025-01-21T06:46:04.805130Z",
     "iopub.status.idle": "2025-01-21T06:46:04.808778Z",
     "shell.execute_reply": "2025-01-21T06:46:04.808205Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.805398Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:59.049286Z",
     "start_time": "2025-03-07T09:44:59.044440Z"
    }
   },
   "source": [
    "32*16"
   ],
   "outputs": [
    {
     "data": {
      "text/plain": [
       "512"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "execution_count": 32
  },
  {
   "cell_type": "code",
   "metadata": {
    "collapsed": false,
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.809881Z",
     "iopub.status.busy": "2025-01-21T06:46:04.809482Z",
     "iopub.status.idle": "2025-01-21T06:46:04.816488Z",
     "shell.execute_reply": "2025-01-21T06:46:04.815987Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.809851Z"
    },
    "jupyter": {
     "outputs_hidden": false
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:59.055548Z",
     "start_time": "2025-03-07T09:44:59.049793Z"
    }
   },
   "source": [
    "#模型总参数量\n",
    "total_params = sum(p.numel() for p in VGG(len(class_names)).parameters() if p.requires_grad)\n",
    "print(f\"Total trainable parameters: {total_params}\")"
   ],
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total trainable parameters: 52266\n"
     ]
    }
   ],
   "execution_count": 33
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.819460Z",
     "iopub.status.busy": "2025-01-21T06:46:04.819152Z",
     "iopub.status.idle": "2025-01-21T06:46:04.846561Z",
     "shell.execute_reply": "2025-01-21T06:46:04.846103Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.819440Z"
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:59.161252Z",
     "start_time": "2025-03-07T09:44:59.056566Z"
    }
   },
   "source": [
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "@torch.no_grad()\n",
    "def evaluating(model, dataloader, loss_fct):\n",
    "    loss_list = []\n",
    "    pred_list = []\n",
    "    label_list = []\n",
    "    for datas, labels in dataloader:\n",
    "        datas = datas.to(device)\n",
    "        labels = labels.to(device)\n",
    "        # 前向计算\n",
    "        logits = model(datas)\n",
    "        loss = loss_fct(logits, labels)         # 验证集损失\n",
    "        loss_list.append(loss.item())\n",
    "        \n",
    "        preds = logits.argmax(axis=-1)    # 验证集预测\n",
    "        pred_list.extend(preds.cpu().numpy().tolist())\n",
    "        label_list.extend(labels.cpu().numpy().tolist())\n",
    "        \n",
    "    acc = accuracy_score(label_list, pred_list)\n",
    "    return np.mean(loss_list), acc\n"
   ],
   "outputs": [],
   "execution_count": 34
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TensorBoard 可视化\n",
    "\n",
    "\n",
    "训练过程中可以使用如下命令启动tensorboard服务。\n",
    "\n",
    "```shell\n",
    "tensorboard \\\n",
    "    --logdir=runs \\     # log 存放路径\n",
    "    --host 0.0.0.0 \\    # ip\n",
    "    --port 8848         # 端口\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.847777Z",
     "iopub.status.busy": "2025-01-21T06:46:04.847313Z",
     "iopub.status.idle": "2025-01-21T06:46:04.907480Z",
     "shell.execute_reply": "2025-01-21T06:46:04.907013Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.847745Z"
    },
    "ExecuteTime": {
     "end_time": "2025-03-07T09:44:59.241039Z",
     "start_time": "2025-03-07T09:44:59.163774Z"
    }
   },
   "source": [
    "from torch.utils.tensorboard import SummaryWriter\n",
    "\n",
    "\n",
    "class TensorBoardCallback:\n",
    "    def __init__(self, log_dir, flush_secs=10):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            log_dir (str): dir to write log.\n",
    "            flush_secs (int, optional): write to dsk each flush_secs seconds. Defaults to 10.\n",
    "        \"\"\"\n",
    "        self.writer = SummaryWriter(log_dir=log_dir, flush_secs=flush_secs)\n",
    "\n",
    "    def draw_model(self, model, input_shape):\n",
    "        self.writer.add_graph(model, input_to_model=torch.randn(input_shape))\n",
    "        \n",
    "    def add_loss_scalars(self, step, loss, val_loss):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/loss\", \n",
    "            tag_scalar_dict={\"loss\": loss, \"val_loss\": val_loss},\n",
    "            global_step=step,\n",
    "            )\n",
    "        \n",
    "    def add_acc_scalars(self, step, acc, val_acc):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/accuracy\",\n",
    "            tag_scalar_dict={\"accuracy\": acc, \"val_accuracy\": val_acc},\n",
    "            global_step=step,\n",
    "        )\n",
    "        \n",
    "    def add_lr_scalars(self, step, learning_rate):\n",
    "        self.writer.add_scalars(\n",
    "            main_tag=\"training/learning_rate\",\n",
    "            tag_scalar_dict={\"learning_rate\": learning_rate},\n",
    "            global_step=step,\n",
    "            \n",
    "        )\n",
    "    \n",
    "    def __call__(self, step, **kwargs):\n",
    "        # add loss\n",
    "        loss = kwargs.pop(\"loss\", None)\n",
    "        val_loss = kwargs.pop(\"val_loss\", None)\n",
    "        if loss is not None and val_loss is not None:\n",
    "            self.add_loss_scalars(step, loss, val_loss)\n",
    "        # add acc\n",
    "        acc = kwargs.pop(\"acc\", None)\n",
    "        val_acc = kwargs.pop(\"val_acc\", None)\n",
    "        if acc is not None and val_acc is not None:\n",
    "            self.add_acc_scalars(step, acc, val_acc)\n",
    "        # add lr\n",
    "        learning_rate = kwargs.pop(\"lr\", None)\n",
    "        if learning_rate is not None:\n",
    "            self.add_lr_scalars(step, learning_rate)\n"
   ],
   "outputs": [
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'tensorboard'",
     "output_type": "error",
     "traceback": [
      "\u001B[1;31m---------------------------------------------------------------------------\u001B[0m",
      "\u001B[1;31mModuleNotFoundError\u001B[0m                       Traceback (most recent call last)",
      "Cell \u001B[1;32mIn[35], line 1\u001B[0m\n\u001B[1;32m----> 1\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mutils\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mtensorboard\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m SummaryWriter\n\u001B[0;32m      4\u001B[0m \u001B[38;5;28;01mclass\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mTensorBoardCallback\u001B[39;00m:\n\u001B[0;32m      5\u001B[0m     \u001B[38;5;28;01mdef\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21m__init__\u001B[39m(\u001B[38;5;28mself\u001B[39m, log_dir, flush_secs\u001B[38;5;241m=\u001B[39m\u001B[38;5;241m10\u001B[39m):\n",
      "File \u001B[1;32m~\\venv\\Lib\\site-packages\\torch\\utils\\tensorboard\\__init__.py:1\u001B[0m\n\u001B[1;32m----> 1\u001B[0m \u001B[38;5;28;01mimport\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtensorboard\u001B[39;00m\n\u001B[0;32m      2\u001B[0m \u001B[38;5;28;01mfrom\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;21;01mtorch\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01m_vendor\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mpackaging\u001B[39;00m\u001B[38;5;21;01m.\u001B[39;00m\u001B[38;5;21;01mversion\u001B[39;00m\u001B[38;5;250m \u001B[39m\u001B[38;5;28;01mimport\u001B[39;00m Version\n\u001B[0;32m      4\u001B[0m \u001B[38;5;28;01mif\u001B[39;00m \u001B[38;5;129;01mnot\u001B[39;00m \u001B[38;5;28mhasattr\u001B[39m(tensorboard, \u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m__version__\u001B[39m\u001B[38;5;124m\"\u001B[39m) \u001B[38;5;129;01mor\u001B[39;00m Version(\n\u001B[0;32m      5\u001B[0m     tensorboard\u001B[38;5;241m.\u001B[39m__version__\n\u001B[0;32m      6\u001B[0m ) \u001B[38;5;241m<\u001B[39m Version(\u001B[38;5;124m\"\u001B[39m\u001B[38;5;124m1.15\u001B[39m\u001B[38;5;124m\"\u001B[39m):\n",
      "\u001B[1;31mModuleNotFoundError\u001B[0m: No module named 'tensorboard'"
     ]
    }
   ],
   "execution_count": 35
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Save Best\n"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.908474Z",
     "iopub.status.busy": "2025-01-21T06:46:04.908064Z",
     "iopub.status.idle": "2025-01-21T06:46:04.913366Z",
     "shell.execute_reply": "2025-01-21T06:46:04.912894Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.908452Z"
    }
   },
   "source": [
    "class SaveCheckpointsCallback:\n",
    "    def __init__(self, save_dir, save_step=5000, save_best_only=True):\n",
    "        \"\"\"\n",
    "        Save checkpoints each save_epoch epoch. \n",
    "        We save checkpoint by epoch in this implementation.\n",
    "        Usually, training scripts with pytorch evaluating model and save checkpoint by step.\n",
    "\n",
    "        Args:\n",
    "            save_dir (str): dir to save checkpoint\n",
    "            save_epoch (int, optional): the frequency to save checkpoint. Defaults to 1.\n",
    "            save_best_only (bool, optional): If True, only save the best model or save each model at every epoch.\n",
    "        \"\"\"\n",
    "        self.save_dir = save_dir\n",
    "        self.save_step = save_step\n",
    "        self.save_best_only = save_best_only\n",
    "        self.best_metrics = -1\n",
    "        \n",
    "        # mkdir\n",
    "        if not os.path.exists(self.save_dir):\n",
    "            os.mkdir(self.save_dir)\n",
    "        \n",
    "    def __call__(self, step, state_dict, metric=None):\n",
    "        if step % self.save_step > 0:\n",
    "            return\n",
    "        \n",
    "        if self.save_best_only:\n",
    "            assert metric is not None\n",
    "            if metric >= self.best_metrics:\n",
    "                # save checkpoints\n",
    "                torch.save(state_dict, os.path.join(self.save_dir, \"best.ckpt\"))\n",
    "                # update best metrics\n",
    "                self.best_metrics = metric\n",
    "        else:\n",
    "            torch.save(state_dict, os.path.join(self.save_dir, f\"{step}.ckpt\"))\n",
    "\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Early Stop"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.914156Z",
     "iopub.status.busy": "2025-01-21T06:46:04.913956Z",
     "iopub.status.idle": "2025-01-21T06:46:04.918260Z",
     "shell.execute_reply": "2025-01-21T06:46:04.917785Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.914137Z"
    }
   },
   "source": [
    "class EarlyStopCallback:\n",
    "    def __init__(self, patience=5, min_delta=0.01):\n",
    "        \"\"\"\n",
    "\n",
    "        Args:\n",
    "            patience (int, optional): Number of epochs with no improvement after which training will be stopped.. Defaults to 5.\n",
    "            min_delta (float, optional): Minimum change in the monitored quantity to qualify as an improvement, i.e. an absolute \n",
    "                change of less than min_delta, will count as no improvement. Defaults to 0.01.\n",
    "        \"\"\"\n",
    "        self.patience = patience\n",
    "        self.min_delta = min_delta\n",
    "        self.best_metric = -1\n",
    "        self.counter = 0\n",
    "        \n",
    "    def __call__(self, metric):\n",
    "        if metric >= self.best_metric + self.min_delta:\n",
    "            # update best metric\n",
    "            self.best_metric = metric\n",
    "            # reset counter \n",
    "            self.counter = 0\n",
    "        else: \n",
    "            self.counter += 1\n",
    "            \n",
    "    @property\n",
    "    def early_stop(self):\n",
    "        return self.counter >= self.patience\n"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:46:04.919099Z",
     "iopub.status.busy": "2025-01-21T06:46:04.918892Z",
     "iopub.status.idle": "2025-01-21T06:47:21.374821Z",
     "shell.execute_reply": "2025-01-21T06:47:21.374251Z",
     "shell.execute_reply.started": "2025-01-21T06:46:04.919070Z"
    },
    "is_executing": true
   },
   "source": [
    "# 训练\n",
    "def training(\n",
    "    model, \n",
    "    train_loader, \n",
    "    val_loader, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    tensorboard_callback=None,\n",
    "    save_ckpt_callback=None,\n",
    "    early_stop_callback=None,\n",
    "    eval_step=500,\n",
    "    ):\n",
    "    record_dict = {\n",
    "        \"train\": [],\n",
    "        \"val\": []\n",
    "    }\n",
    "    \n",
    "    global_step = 0\n",
    "    model.train()\n",
    "    with tqdm(total=epoch * len(train_loader)) as pbar:\n",
    "        for epoch_id in range(epoch):\n",
    "            # training\n",
    "            for datas, labels in train_loader:\n",
    "                datas = datas.to(device)\n",
    "                labels = labels.to(device)\n",
    "                # 梯度清空\n",
    "                optimizer.zero_grad()\n",
    "                # 模型前向计算\n",
    "                logits = model(datas)\n",
    "                # 计算损失\n",
    "                loss = loss_fct(logits, labels)\n",
    "                # 梯度回传\n",
    "                loss.backward()\n",
    "                # 调整优化器，包括学习率的变动等\n",
    "                optimizer.step()\n",
    "                preds = logits.argmax(axis=-1)\n",
    "            \n",
    "                acc = accuracy_score(labels.cpu().numpy(), preds.cpu().numpy())    \n",
    "                loss = loss.cpu().item()\n",
    "                # record\n",
    "                \n",
    "                record_dict[\"train\"].append({\n",
    "                    \"loss\": loss, \"acc\": acc, \"step\": global_step\n",
    "                })\n",
    "                \n",
    "                # evaluating\n",
    "                if global_step % eval_step == 0:\n",
    "                    model.eval()\n",
    "                    val_loss, val_acc = evaluating(model, val_loader, loss_fct)\n",
    "                    record_dict[\"val\"].append({\n",
    "                        \"loss\": val_loss, \"acc\": val_acc, \"step\": global_step\n",
    "                    })\n",
    "                    model.train()\n",
    "                    \n",
    "                    # 1. 使用 tensorboard 可视化\n",
    "                    if tensorboard_callback is not None:\n",
    "                        tensorboard_callback(\n",
    "                            global_step, \n",
    "                            loss=loss, val_loss=val_loss,\n",
    "                            acc=acc, val_acc=val_acc,\n",
    "                            lr=optimizer.param_groups[0][\"lr\"],\n",
    "                            )\n",
    "                \n",
    "                    # 2. 保存模型权重 save model checkpoint\n",
    "                    if save_ckpt_callback is not None:\n",
    "                        save_ckpt_callback(global_step, model.state_dict(), metric=val_acc)\n",
    "\n",
    "                    # 3. 早停 Early Stop\n",
    "                    if early_stop_callback is not None:\n",
    "                        early_stop_callback(val_acc)\n",
    "                        if early_stop_callback.early_stop:\n",
    "                            print(f\"Early stop at epoch {epoch_id} / global_step {global_step}\")\n",
    "                            return record_dict\n",
    "                    \n",
    "                # udate step\n",
    "                global_step += 1\n",
    "                pbar.update(1)\n",
    "                pbar.set_postfix({\"epoch\": epoch_id})\n",
    "        \n",
    "    return record_dict\n",
    "        \n",
    "\n",
    "epoch = 10\n",
    "\n",
    "model = VGG(num_classes=10)\n",
    "\n",
    "# 1. 定义损失函数 采用交叉熵损失\n",
    "loss_fct = nn.CrossEntropyLoss()\n",
    "# 2. 定义优化器 采用 adam\n",
    "# Optimizers specified in the torch.optim package\n",
    "#可以修改adam里边的weight_decay参数，来控制L2正则化的程度,beta1和beta2是指数衰减率\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)\n",
    "\n",
    "# 1. tensorboard 可视化\n",
    "if not os.path.exists(\"runs\"):\n",
    "    os.mkdir(\"runs\")\n",
    "tensorboard_callback = TensorBoardCallback(\"runs/vgg\")\n",
    "tensorboard_callback.draw_model(model, [1, 3, IMAGE_SIZE, IMAGE_SIZE])\n",
    "# 2. save best\n",
    "if not os.path.exists(\"checkpoints\"):\n",
    "    os.makedirs(\"checkpoints\")\n",
    "save_ckpt_callback = SaveCheckpointsCallback(\"checkpoints/vgg\", save_step=len(train_dl), save_best_only=True)\n",
    "# 3. early stop\n",
    "early_stop_callback = EarlyStopCallback(patience=5)\n",
    "\n",
    "model = model.to(device)\n",
    "record = training(\n",
    "    model, \n",
    "    train_dl, \n",
    "    eval_dl, \n",
    "    epoch, \n",
    "    loss_fct, \n",
    "    optimizer, \n",
    "    tensorboard_callback=None,\n",
    "    save_ckpt_callback=save_ckpt_callback,\n",
    "    early_stop_callback=early_stop_callback,\n",
    "    eval_step=len(train_dl)\n",
    "    )"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:47:21.375822Z",
     "iopub.status.busy": "2025-01-21T06:47:21.375549Z",
     "iopub.status.idle": "2025-01-21T06:47:21.624424Z",
     "shell.execute_reply": "2025-01-21T06:47:21.623939Z",
     "shell.execute_reply.started": "2025-01-21T06:47:21.375799Z"
    },
    "is_executing": true
   },
   "source": [
    "#画线要注意的是损失是不一定在零到1之间的\n",
    "def plot_learning_curves(record_dict, sample_step=500):\n",
    "    # build DataFrame\n",
    "    train_df = pd.DataFrame(record_dict[\"train\"]).set_index(\"step\").iloc[::sample_step]\n",
    "    val_df = pd.DataFrame(record_dict[\"val\"]).set_index(\"step\")\n",
    "\n",
    "    # plot\n",
    "    fig_num = len(train_df.columns)\n",
    "    fig, axs = plt.subplots(1, fig_num, figsize=(5 * fig_num, 5))\n",
    "    for idx, item in enumerate(train_df.columns):    \n",
    "        axs[idx].plot(train_df.index, train_df[item], label=f\"train_{item}\")\n",
    "        axs[idx].plot(val_df.index, val_df[item], label=f\"val_{item}\")\n",
    "        axs[idx].grid()\n",
    "        axs[idx].legend()\n",
    "        # axs[idx].set_xticks(range(0, train_df.index[-1], 5000))\n",
    "        # axs[idx].set_xticklabels(map(lambda x: f\"{int(x/1000)}k\", range(0, train_df.index[-1], 5000)))\n",
    "        axs[idx].set_xlabel(\"step\")\n",
    "    \n",
    "    plt.show()\n",
    "\n",
    "plot_learning_curves(record, sample_step=100)  #横坐标是 steps"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 评估"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:47:21.625322Z",
     "iopub.status.busy": "2025-01-21T06:47:21.625007Z",
     "iopub.status.idle": "2025-01-21T06:47:22.290255Z",
     "shell.execute_reply": "2025-01-21T06:47:22.289643Z",
     "shell.execute_reply.started": "2025-01-21T06:47:21.625301Z"
    },
    "is_executing": true
   },
   "source": [
    "# dataload for evaluating\n",
    "\n",
    "# load checkpoints\n",
    "model.load_state_dict(torch.load(\"checkpoints/vgg/best.ckpt\", map_location=\"cpu\"))\n",
    "\n",
    "model.eval()\n",
    "loss, acc = evaluating(model, eval_dl, loss_fct)\n",
    "print(f\"loss:     {loss:.4f}\\naccuracy: {acc:.4f}\")"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 推理"
   ]
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:47:22.291308Z",
     "iopub.status.busy": "2025-01-21T06:47:22.290944Z",
     "iopub.status.idle": "2025-01-21T06:47:22.293881Z",
     "shell.execute_reply": "2025-01-21T06:47:22.293440Z",
     "shell.execute_reply.started": "2025-01-21T06:47:22.291285Z"
    },
    "is_executing": true
   },
   "source": [
    "# # test_df\n",
    "# test_ds = Cifar10Dataset(\"test\", transform=transforms_eval)\n",
    "# test_dl = DataLoader(test_ds, batch_size=batch_size, shuffle=False, drop_last=False)\n",
    "#\n",
    "# preds_collect = []\n",
    "# model.eval()\n",
    "# for data, fake_label in tqdm(test_dl):\n",
    "#     data = data.to(device=device)\n",
    "#     logits = model(data)\n",
    "#     preds = [test_ds.idx_to_label[idx] for idx in logits.argmax(axis=-1).cpu().tolist()]\n",
    "#     preds_collect.extend(preds)\n",
    "#\n",
    "# test_df[\"class\"] = preds_collect\n",
    "# test_df.head()"
   ],
   "outputs": [],
   "execution_count": null
  },
  {
   "cell_type": "code",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2025-01-21T06:47:22.294784Z",
     "iopub.status.busy": "2025-01-21T06:47:22.294481Z",
     "iopub.status.idle": "2025-01-21T06:47:22.296790Z",
     "shell.execute_reply": "2025-01-21T06:47:22.296361Z",
     "shell.execute_reply.started": "2025-01-21T06:47:22.294764Z"
    },
    "is_executing": true,
    "tags": []
   },
   "source": [
    "# # 导出 submission.csv\n",
    "# test_df.to_csv(\"submission.csv\", index=False)"
   ],
   "outputs": [],
   "execution_count": null
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [
    {
     "databundleVersionId": 46718,
     "sourceId": 3649,
     "sourceType": "competition"
    }
   ],
   "dockerImageVersionId": 30588,
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
